# -*- coding:utf8 -*-
#!/usr/bin/env python
import os
import feedparser
import copy
import rfc822
import time
import datetime
import HTMLParser
import urllib
import lxml.html
import chardet
import threading
from unicodedata import normalize

from django.core.management.base import BaseCommand, CommandError
from django.db.models import Q
from django.utils.encoding import smart_str, smart_unicode

from feedsaver.saver.models import NewsArchive

from lxml import etree

def rawr(begin,limit,subject,self):
    self.stdout.write('>> Starting main loop.\n')

    for i in range(begin,limit):

        ncount = NewsArchive.objects.filter(subject=subject).count()

        #numero de noticias salva por assunto

        if ncount >= 100:
            return
  
        url = "http://br.noticias.yahoo.com/"+ subject +"/arquivo/"+ str(i) +".html"
        
        content = urllib.urlopen(url).read()
        
        doc = lxml.html.fromstring(content, base_url=url)

        doc.make_links_absolute()

        for link in doc.cssselect('h4 a'):
            link_content = smart_str(link.text_content(), encoding='utf-8', strings_only=False, errors='strict')
            link_href = smart_str(link.get('href'), encoding='utf-8', strings_only=False, errors='strict')
            self.stdout.write(link_content)
     
            self.stdout.write('\n\n')

            content = urllib.urlopen(link_href).read()

            doc = lxml.html.fromstring(content)

            text = ""
                
            for p in doc.cssselect('div.yom-art-content div.bd p'):
                #text += normalize('NFKD', p.text_content().decode('latin')).encode('utf-8', 'ignore')
                #text += smart_unicode(p.text_content(), encoding='utf-8', strings_only=False, errors='strict')
                tmp = p.text_content()
                tmp = tmp.lower()
                tmp = tmp.replace(',','')
                tmp = tmp.replace('.',' ')
                tmp = tmp.replace("'",'')
                tmp = tmp.replace('"','')
                tmp = tmp.replace('!','')
                tmp = tmp.replace(';','')
                tmp = tmp.replace(':','')
                tmp = tmp.replace('(','')
                tmp = tmp.replace(')','')
                
                text += smart_str(tmp, encoding='utf-8', strings_only=True, errors='strict')
                text += '\n'
       
            if text.__len__() < 150:
                self.stdout.write('texto muito pequeno\n')
                text = ""

            else:
                
                # See if the text already exists in DB
                title = smart_str(link.text_content(), encoding='utf-8', strings_only=False, errors='strict')
                exists = NewsArchive.objects.filter(title=title).count()

                self.stdout.write(subject+'\n')
                if exists > 0:
                    self.stdout.write('Notícia já existente\n')

                else:
                    # Save the text
                    n = NewsArchive()

                    n.title = link.text_content()
                    n.text = text

                    n.subject = subject
                    try:
                        n.save()
                        self.stdout.write('Notícia salva!\n')

                    except:
                        self.stdout.write('erro!\n')
                        pass

            self.stdout.write(str(NewsArchive.objects.filter(subject=subject).count()))
            self.stdout.write('\n\n')

class thread_brasil ( threading.Thread ):
    def run ( self ):
        rawr(0,100,'brasil',command)

class thread_mundo ( threading.Thread ):
    def run ( self ):
        rawr(0,100,'mundo',command)

class thread_tecnologia ( threading.Thread ):
    def run ( self ):
        rawr(0,100,'tecnologia',command)

class thread_ciencia ( threading.Thread ):
    def run ( self ):
        rawr(0,100,'ciencia',command)

class thread_entretenimento ( threading.Thread ):
    def run ( self ):
        rawr(0,100,'entretenimento',command)

class thread_economia ( threading.Thread ):
    def run ( self ):
        rawr(200,300,'economia',command)

class thread_economia2 ( threading.Thread ):
    def run ( self ):
        rawr(300,400,'economia',command)

class Command(BaseCommand):
    args = 'no args'
    help = 'Extract info from CPR'
    
    def handle(self, *args, **options):

        global command
        command = self

        #thread_tecnologia().start()
        #thread_ciencia().start()
        #thread_entretenimento().start()

        thread_economia().start()
    	thread_brasil().start()	
        thread_mundo().start()
